// panda.cpp : Defines the entry point for the console application.
//

#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <algorithm>
#include <iterator>
#include <boost/smart_ptr.hpp>
#define _MY_STL_MACROS
#include <stlhelper.h>
#include <set_mark_diff.h"
#include <assert.h>
#include <deque>


#include "../panda/remove_dependent_orphans.h"
#include "../panda/transfer_data_to_postgres.h"
#include "../panda/components/bcpp_parsefasta.h"
#include "../panda/components/pg_db_cursor.h"
#include "../panda/panda.h"
#include "../panda/components/libpq++/libpq++.h"
#include "../panda/components/progress_indicator.h"
#include "../panda/components/libpq++/pgdatabase.h"
#include "../panda/components/portable_timer.h"
#include "get_arg.h"

using std::back_inserter;
using std::ofstream;
using std::ifstream;
using std::vector;
using std::ostringstream;
using std::sort;
using std::cout;
using std::string;
using std::ostream;
using std::deque;
using std::less;





/*
Function call hierachy
main
	remove_orphaned_sequences
		transfer_sequence_crc_to_pg
			is_temp_file_open
			is_temp_file_written

*/





//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//	Description and enums for the statistics and durations from different elements
//		of the entire process

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
char* Results_Times_Descriptions[] ={
	"Counting sequences ('>'s) from the FASTA file.",				//  0
    "Retrieving current accessions hashes from Panda.",				//  1
	"Parsing accessions and sequences from the FASTA file.",		//  2
	"Importing PIs for obsolete accessions into Panda.",            //  3
	"Archiving and removing obsolete accessions from Panda.",       //  4
	"Importing new accessions into PostgreSQL.",					//  5
    "PostgreSQL recreating database indices for the accessions.",	//  6
	"Importing new secondary IDs into PostgreSQL.",					//  7
    "PostgreSQL recreating database indices for the secondary IDs.",//  8
	"Retrieving from Panda the relationships for gi & TaxIDs.",     //  9
	"Retrieving from Panda the relationships for tax names & IDs.", // 10
	"Getting all sequences PIs currently in Panda.",                // 11
	"Comparing newly parsed and existing sequences PIs.",			// 12
	"Checking for PI hash collisions.",                             // 13
	"Importing sequences into PostgreSQL.",							// 14
	"PostgreSQL recreating indices for the 'sequences' table.",		// 15
	"Retrieve all unprocessed novel PIs from Panda.",               // 16
	"Add new PIs to the list of unprocessed sequences in Panda.",   // 17
	"Exporting PIs of all accessions from Panda.",					// 18
	"Comparing accession and sequence PIs.",						// 19
	"Removing obsolete sequences in Panda.",						// 20
    "Removing orphaned entries linked to obsolete sequences.",		// 21
    "PostgreSQL re-collecting database statistics.",				// 22
    "Total Time.",													// 23
    "END"															// 24
};

enum eRESULTS_TIMES
{
	TIME_CNT_SEQ,													//  0
	TIME_PG_IMPORT_ACC_HASH,                                        //  1
	TIME_PARSE_ACC,													//  2
	TIME_PG_IMPORT_OBSOLETE_ACC_PI,                                 //  3
	TIME_PG_REMOVE_OBSOLETE_ACC,									//  4
	TIME_PG_ADD_ACC,												//  5
	TIME_PG_RECREATE_ACC_INDICES,                                   //  6
	TIME_PG_ADD_SECIDS,                                             //  7
	TIME_PG_RECREATE_SECIDS_INDICES,                                //  8
	TIME_PG_IMPORT_GI_TAXID_LOOKUP,                                 //  9
	TIME_PG_IMPORT_TAXNAME_TAXID_LOOKUP,                            // 10
	TIME_PG_IMPORT_PANDA_SEQ_PI,                                    // 11
	TIME_COMPARE_SEQ_PI,											// 12
	TIME_CHECK_PI_COLLISIONS,                                       // 13
	TIME_PG_ADD_SEQ,												// 14
	TIME_PG_RECREATE_SEQ_INDICES,									// 15
	TIME_PG_EXPORT_NEW_SEQ_PI,                                      // 16
	TIME_PG_ADD_NEW_SEQ_PI,                                         // 17
	TIME_PG_EXPORT_ACC_PI,											// 18
	TIME_COMPARE_ACC_SEQ_PI,										// 19
	TIME_PG_REMOVE_ORPHANED_SEQ,									// 20
	TIME_PG_SYNC_PI_DEPENDENTS,										// 21
	TIME_PG_VACUUM,													// 22
	TIME_TOTAL,														// 23
	TIME_END														// 24
};


unsigned SAVED_TIMES[] =
{
	TIME_CNT_SEQ,													//  0
	TIME_PG_IMPORT_ACC_HASH,                                        //  1
	TIME_PARSE_ACC,													//  2
	TIME_PG_IMPORT_OBSOLETE_ACC_PI,                                 //  3
	TIME_PG_REMOVE_OBSOLETE_ACC,									//  4
	TIME_PG_ADD_ACC,												//  5
	TIME_PG_RECREATE_ACC_INDICES,                                   //  6
	TIME_PG_ADD_SECIDS,                                             //  7
	TIME_PG_RECREATE_SECIDS_INDICES,                                //  8
	TIME_PG_IMPORT_PANDA_SEQ_PI,                                    //  9
	TIME_COMPARE_SEQ_PI,											// 10
	TIME_CHECK_PI_COLLISIONS,                                       // 11
	TIME_PG_ADD_SEQ,												// 12
	TIME_PG_RECREATE_SEQ_INDICES,									// 13
	TIME_PG_EXPORT_NEW_SEQ_PI,                                      // 14
	TIME_PG_ADD_NEW_SEQ_PI,                                         // 15
	TIME_PG_EXPORT_ACC_PI,											// 16
	TIME_COMPARE_ACC_SEQ_PI,										// 17
	TIME_PG_REMOVE_ORPHANED_SEQ,									// 18
	TIME_PG_SYNC_PI_DEPENDENTS,										// 19
	TIME_PG_VACUUM,													// 20
	TIME_TOTAL														// 21
};



char* Results_Counts_Descriptions[] ={
    "sequences parsed from the FASTA file.",						//  0
	"sequences without any taxa specified.",						//  1
    "sequences without a matching accession code.",					//  2
    "duplicate sequences in the FASTA file.",						//  3
	"matching accessions contained in the PostgreSQL database.",	//  4
    "obsolete accessions removed from the PostgreSQL database.",	//  5
    "new accessions added to the PostgreSQL database.",				//  6
    "new secondary IDs added to the PostgreSQL database.",			//  7
	"sequences contained in the PostgreSQL database.",				//  8
    "obsolete sequences removed from the PostgreSQL database.",		//  9
	"new sequences added to the PostgreSQL database.",				// 10
	"END"															// 11
};																	


enum eRESULTS_COUNTS
{
	CNT_FASTA_SEQ_PARSED,											//  0
	CNT_FASTA_ACC_SANS_TAXA,                                        //  1
	CNT_FASTA_ACC_SANS_CODES,                                       //  2
	CNT_FASTA_PI_DUPLICATES,	                                    //  3
	CNT_ACC_SAME_ORIGIN_IN_PANDA,                                   //  4
	CNT_ACC_OBSOLETE,                                               //  5
	CNT_ACC_ADDITIONAL,			                                    //  6
	CNT_SECIDS_ADDITIONAL,                                          //  7
	CNT_SEQ_IN_PANDA,                                               //  8
	CNT_SEQ_OBSOLETE,                                               //  9
	CNT_SEQ_ADDITIONAL,                                             // 10
	CNT_END						                                    // 11
};


//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//	t_program_results

//		statistics from and the duration of each element of the process

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
struct t_program_results
{
	float		times [TIME_END];
	unsigned	counts [CNT_END];
	t_program_results()
	{
		memset(times, 0, sizeof (float) * TIME_END);
		memset(counts, 0, sizeof (unsigned) * CNT_END);
	}
	string GetCSV() const
	{
		ostringstream os;

		// write out counts with descriptions
		for (unsigned i = 0; i < CNT_END; ++i)
			os << counts[i] << ", ";

		// timing info printed out at +/- 0.1 s
		os << std::setprecision(1) << std::fixed;
			
		// write out times with descriptions
		
		// first value
		os << times[SAVED_TIMES[0]];
		
		for (unsigned i = 1; i < sizeof(SAVED_TIMES) / sizeof(SAVED_TIMES[0]); ++i)
			os << "," << times[SAVED_TIMES[i]];
		
		return(os.str());
	}
};

template <typename ostrm>
ostrm& operator << (ostrm& os, const t_program_results& results)
{
	assert (!strcmp(Results_Counts_Descriptions[CNT_END], "END"));
	assert (!strcmp(Results_Times_Descriptions[TIME_END], "END"));

	// write out counts with descriptions
	for (unsigned i = 0; i < CNT_END; ++i)
		os << '\t'	<< results.counts[i] << "\t "
					<< Results_Counts_Descriptions[i] << "\n";

	// timing info printed out at +/- 0.1 s
	os << std::setprecision(1) << std::fixed;
		
	// write out times with descriptions	
	for (unsigned i = 0; i < TIME_END; ++i)
		if (results.times[i] > 4.0)
			os	<< '\t' << results.times[i] << "s\t: "
				<< Results_Times_Descriptions[i] << "\n";
	os << "\n";

	// make sure worked
	assert(os.good());

	return os;
}





//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//	Current_operation

//		statistics from and the duration of each element of the process

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
namespace
{
	
	class Current_operation
	{
		eRESULTS_TIMES		op_id;
		ostream&			VERBOSE;
		t_program_results&	results;
		t_portable_timer	timer;
	public:
		Current_operation (eRESULTS_TIMES op_id_, ostream&	VERBOSE_,
										t_program_results& results_):
			op_id (op_id_), VERBOSE(VERBOSE_), results(results_)
		{
			assert(op_id < (sizeof Results_Times_Descriptions / sizeof(char*)));
			VERBOSE << '\t' << Results_Times_Descriptions[op_id] << ".." << std::endl;
		}
		
		~Current_operation()
		{
			results.times[op_id] = timer.reset();
		}
		
	};
}



//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//	t_auto_log_results

//		writes programme statistics to the error log automatically on
//			premature programme exit

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
struct t_auto_log_results
{
	t_program_results&	results;
	ostream&			CLOG;
	bool				do_log;
	t_auto_log_results(t_program_results& r, ostream& l):
								results(r), CLOG(l), do_log(true) {}
	~t_auto_log_results(){if (do_log) CLOG << results; }
	void abandon_log(){ do_log = false;	}
};







//SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS
//		remove_obsolete_sequences

//SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS
bool remove_obsolete_sequences(	t_pg_cursor& 			pg_db,
								deque<t_pi>&			seq_pi_all,
								t_program_args& 		args,
								t_program_results& 		results)
{
	deque<t_pi>			acc_pi_all;
	args.VERBOSE() << "\tRemoving obsolete sequences...\n";
	//____________________________________________________________________________________
	// 		Exporting PIs of all accessions from PostgreSQL.
	//____________________________________________________________________________________
	{
		Current_operation op(TIME_PG_EXPORT_ACC_PI, args.VERBOSE(), results);
		//					 ^^^^^^^^^^^^^^^^^^^^^
		if (!pg_import_PIs_from_table(pg_db, "accessions", acc_pi_all,
										0,
										args.VERBOSE(),	args.CERR().strm()))
			return false;
	}

	//____________________________________________________________________________________
	// 		Comparing accession and sequence PIs.
	//				Get sequence PIs that are not in accessions
	//				i.e. do a set difference
	//____________________________________________________________________________________
	deque<t_pi> seq_pi_obsolete;
	
	{
		Current_operation op(TIME_COMPARE_ACC_SEQ_PI, args.VERBOSE(), results);
		//					 ^^^^^^^^^^^^^^^^^^^^^^^
		// sort and uniquify
		assert(stlhelper::is_sorted(beg_to_end(acc_pi_all)));
		assert(stlhelper::is_unique(beg_to_end(acc_pi_all)));

		assert(stlhelper::is_sorted(beg_to_end(seq_pi_all)));
		
/*	DEBUG
		std::cerr << string (60, '*') << "\n";
		for (unsigned i = 0; i < seq_pi_all.size(); ++i)
			std::cerr<< seq_pi_all[i] << "\n";
		std::cerr << string (60, '*') << "\n";
		for (unsigned i = 0; i < acc_pi_all.size(); ++i)
			std::cerr<< acc_pi_all[i] << "\n";
		std::cerr << string (60, '*') << "\n";
*/

		deque<t_pi> acc_pi_missing;
		set_mark_diff(	beg_to_end(seq_pi_all),
						beg_to_end(acc_pi_all),
						less<t_pi>(),
						op_save_in_deque<t_pi>(seq_pi_obsolete),
						op_save_in_deque<t_pi>(acc_pi_missing));


		// print out diagnostic if any accessions with invalid PIs
		if (acc_pi_missing.size())
		{
			args.CERR().strm()
						<< "\nWARNING!! "
						<< acc_pi_missing.size()
						<< " accesssions with PIs not from any sequence!!\n\n";
		}

		// remove obsolete pis from list of all extant pis
		deque<t_pi> seq_pi_all_sans_obsolete;

		set_difference(	beg_to_end(seq_pi_all),
						beg_to_end(seq_pi_obsolete),
						back_inserter(seq_pi_all_sans_obsolete));
		seq_pi_all_sans_obsolete.swap(seq_pi_all);


		// count orphans	
		results.counts[CNT_SEQ_OBSOLETE] = seq_pi_obsolete.size();
	}
	

	//____________________________________________________________________________________
	// 		PostgreSQL removing orphaned sequences.
	//____________________________________________________________________________________
	{
		Current_operation op(TIME_PG_REMOVE_ORPHANED_SEQ, args.VERBOSE(), results);
		//					 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
		// transfer PIs of orphaned sequences to file
		if (!pg_remove_PIs_from_table(pg_db,
									  "sequences",
									  seq_pi_obsolete,
									  args.VERBOSE(),
									  args.CERR().strm(),
									  "len,seq_hash,sequence,seq_date_added"))
			return false;
			
	}

	return true;

}
















//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//		Main

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
int main(int argc, char* argv[])
{
	t_portable_timer timer;
	
	// don't use C streams
	std::ios::sync_with_stdio(false);

	// saving parameters
	string exe_parameters;
	for (unsigned i = 0; i < argc; ++i)
		exe_parameters += string(argv[i]) + ' ';

	// we use cout to print progress so flush per call
	cout.setf(std::ios_base::unitbuf);

	// get command line arguments
	t_program_args args;
	if (!GetArguments(argc, argv, args))
		return 1;

	// write results to the error log automatically on failure / errors
	t_program_results results;
	dual_streams CLOG = args.CLOG();
	t_auto_log_results auto_log_results(results, CLOG.strm());

	//____________________________________________________________________________________
	// 		Connect to PostgreSQL
	//____________________________________________________________________________________
	t_pg_cursor pg_db(args.db_login.c_str(), "panda_cursor");	//"dbname=test user=postgres"
	if ( pg_db.ConnectionBad() )
	{
		args.CERR().strm()	<< "\nERROR!! Connection to database ["
							<< args.db_login << "] failed.\n"
							<< pg_db.ErrorMessage() << "\n\n";
		return 1;
	}
	

	//____________________________________________________________________________________
	// 		Run all specified SQL commands.
	//____________________________________________________________________________________
	{
		args.VERBOSE() << "\tRunning the SQL commands...\n";
		t_progress_indicator dots(args.VERBOSE(), 1);
		for (unsigned i = 0; i < args.sql_commands.size(); ++i)
		{
			if( pg_db.ExecCommandOk(args.sql_commands[i].c_str()))
				dots++;
			else
			{
				dots.finish();
				args.CERR().strm()
							<< "\nERROR!!"
							<< "There was an error executing the following SQL command:\n"
							<< "SQL [ " << args.sql_commands[i] << " ]\n"
							<< pg_db.ErrorMessage();
				return 1;
			}
		}
	}
	
	args.VERBOSE() << "    Remove sequences no longer referenced by any accessions...\n";

	//____________________________________________________________________________________
	// 		Get PIs for all sequences from postgreSQL
	//____________________________________________________________________________________
	deque<t_pi>			seq_PIs;
	{
		args.VERBOSE() << "\tImporting PIs for all sequences...\n";
		if (!pg_import_PIs_from_table(pg_db, "sequences", seq_PIs,
										0,
										args.VERBOSE(),	args.CERR().strm()))
			return 1;
		results.counts[CNT_SEQ_IN_PANDA] = seq_PIs.size();
	}
	


	//____________________________________________________________________________________
	// 		Remove orphans
	//____________________________________________________________________________________
	{
		// remove sequence orphans
		if(!remove_obsolete_sequences(pg_db, seq_PIs, args, results))
				return 1;
	}

	//____________________________________________________________________________________
	// 		Synching tables dependent on PIs
	//____________________________________________________________________________________
	{
		Current_operation op(TIME_PG_SYNC_PI_DEPENDENTS, args.IGNORE(), results);
		args.VERBOSE() << "    " << Results_Times_Descriptions[TIME_PG_SYNC_PI_DEPENDENTS]
					   << ".." << std::endl;
		// for each dependent table remove orphans with no corresponding sequences
		if (!synchronize_sequence_dependent_tables(pg_db, seq_PIs, args.VERBOSE(),
														args.CERR().strm()))
			return 1;
	
	}
	
	
	//____________________________________________________________________________________
	// 		PostgreSQL re-collecting database statistics.
	// 			commit transaction and reanalyse database
	//____________________________________________________________________________________
	{
		Current_operation op(TIME_PG_VACUUM, args.VERBOSE(), results);
		// commit transaction
		pg_db.Commit();

		if (args.vacuum)
		{
			if( !pg_db.ExecCommandOk("ANALYZE sequences") ||
				!pg_db.ExecCommandOk("ANALYZE accessions"))
				args.CERR().strm()	<< "ERROR!! Failed to clean and analyse the new database tables\n"
									<< pg_db.ErrorMessage();
		}
	}
	

	
	//____________________________________________________________________________________
	// 		save results / timing statistics
	//____________________________________________________________________________________
	//
	{
		string SQL_command = "INSERT INTO updates_log VALUES('"
								+ args.description	+ "',"
								+ results.GetCSV()			+ ",'"
								+ exe_parameters			+ "')";
		if( !pg_db.ExecCommandOk(SQL_command.c_str()))
			args.CERR().strm()
			<< "ERROR!! Failed to log update statistics and timings into PostgreSQL\n\n"
			<< SQL_command << "\n"
			<< pg_db.ErrorMessage();
	}
	
	
	// succeeded: no need to write results to error log
	// print results / timing statistics only if verbose
	auto_log_results.abandon_log();
	args.VERBOSE() << "\n";
	args.VERBOSE() << "    " << string(60, '*') << "\n";
	args.VERBOSE() << results;
	args.VERBOSE() << "    " << string(60, '*') << "\n";

	args.CLOG().strm() << "Finished processing SQL Commands [" << args.description << "]\n";

	return 0;
}


